import logging
import os
import os.path
from concurrent.futures.thread import ThreadPoolExecutor

import requests

from cn.boz.utils.ProjectUtils import ProjectUtils
from cn.boz.utils.datatheif import mongodbquery

logging.basicConfig(level=logging.INFO,
                    format="%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")


class DataTheif:
    """
    数据贼
    """

    def __init__(self):
        self.mop = mongodbquery.MongoOperator()
        self.overwriteProduct = False
        self.overwritePages = False
        self.overwriteImage = False
        self.baseimgurl = "https://aidi-public.oss-cn-zhangjiakou.aliyuncs.com/"
        self.baseweburl = "https://merchant.qifengcm.com"
        self.storeBase = ProjectUtils.getConfigSessionValue("datatheif", "storetarget")
        self.header = {
            'clothing-token':
                'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJ7XCJjbGllbnRJZFwiOjEwMDM4LFwidXNlcklkXCI6NDcxfSIsImlhdCI6MTYzNTA2Mzk3MiwiZXhwIjoxNjM1MDY3NTcyfQ.uMfi8R06BbSdJ6mmv9GEZKR1SUJkgphnys6IVcRNyjfM-uqpGzP8izBjvvZEjn6r0XKfJ9C4Bs3BgnBDwKGBpA'
        }
        self.executor = ThreadPoolExecutor(max_workers=15, thread_name_prefix='data_theif_')

    def theifForCategory(self):
        """

        :return:
        """
        categoryUrl = "%s/merchant/productCategory" % (self.baseweburl)
        self.theifForUrlResources(categoryUrl, useHeader=True, forceUpdate=True)
        pass

    def theifForProduct(self, res):
        """
         对单个商品进行数据获取
        """
        item = res.json()

        data = item['data']

        if item['resultCode'] != 200:
            raise Exception(item['description'])

        detailPictureList = data['detailPictureList']
        mainPictureList = data['mainPictureList']
        skuList = data['skuList']

        for detailPicture in detailPictureList:
            logging.debug('fetching detail picture:%s' % (detailPicture))
            self.theifForUrlResources(detailPicture)

        for mainPicture in mainPictureList:
            logging.debug("fetching main picture:%s" % (mainPicture))
            self.theifForUrlResources(mainPicture)

        for sku in skuList:
            picture = sku['picture']
            logging.debug("fetching sku :%s" % (picture))
            self.theifForUrlResources(picture)

    """
    按照分类类别进行查询数据
    """

    def productByType(self):
        pids = self.mop.queryAllSubPid();
        ids = []
        for pid in pids:
            url = "https://merchant.qifengcm.com/merchant/product?pageSize=200&orderBy=&productCategoryId=%d&productName=&tag=&pageIndex=1"
            self.theifForUrlResources(url % (pid), useHeader=True, forceUpdate=True)
        pass

    def parseUrl(self, url):
        storepos = url[len('https://'):][url[len('https://'):].index("/"):]
        storepath = storepos[:storepos.rindex("/")]
        storename = storepos[storepos.rindex("/"):]
        return {
            'storepath': storepath,
            'storename': storename,
        }
        pass

    def theifForUrlResources(self, url, callback=None, useHeader=False, forceUpdate=False):
        """
        这里只是对指定URL的内容进行抓取，设计上是把回调函数送入，方便对获取到数据之后，再进行其他的操作！
        """
        result = self.parseUrl(url)
        storepath = self.storeBase + result['storepath']
        storename = result['storename']
        storefullname = "%s%s" % (storepath, storename)
        if not os.path.exists(storepath):
            os.makedirs(storepath)
        if os.path.isfile(storefullname) and not forceUpdate:
            logging.debug("file:%s already exists!" % (storefullname))
            return

        # 使用多线程
        future = self.executor.submit(self.doForUrlResources, callback, storefullname, storename, url, useHeader)
        return future
        # self.doForUrlResources(callback, storefullname, storename, url, useHeader)

    def doForUrlResources(self, callback, storefullname, storename, url, useHeader):
        if useHeader:
            res = requests.get(url, headers=self.header)
        else:
            res = requests.get(url)
        fp = open(storefullname, 'wb+')
        fp.write(res.content)
        fp.close()
        logging.info("file %s write success" % (storefullname))
        if callback:
            callback(res)

    def productListCallback(self, res):
        """
        送入产品列表，然后对产品列表进行获取
        res是送入的数据结构
        """
        item = res.json();
        data = item['data']
        resultCode = item['resultCode']
        if resultCode != 200:
            raise Exception(item['description'])

        records = data['records']
        for record in records:
            # 抓取图片以及
            productId = record['productId'];
            self.theifForProductById(productId)
        pass

    def theifForProductById(self, productId):
        # 抓取资源
        logging.debug('fetching for procudt:%s' % (productId))
        producturl = "%s/merchant/product/%s" % (self.baseweburl, productId)
        feature = self.theifForUrlResources(producturl, self.theifForProduct, useHeader=True, forceUpdate=True)
        feature.result()

    def theifForProductList(self, pageidx, pagesize=200, withresource=True):
        """
        指定页pageidx,指定页大小pagesize
        首先是请求分页数据，即0-200页面基本数据，然后，根据页面的详细数据，请求指定ID的数据信息，
        最后将指定ID的信息进行保存,再最后，对指定ID的商品的图片进行抓取
        withresource表示抓取里面的资源，以送入的callback位基准
        """
        logging.info("fetching for page:%d " % (pageidx))
        url = "%s/merchant/product?pageSize=%d&orderBy=&productCategoryId=&productName=&tag=&pageIndex=%d" \
              % (self.baseweburl, pagesize, pageidx)
        if withresource:
            future = self.theifForUrlResources(url, useHeader=True, callback=self.productListCallback, forceUpdate=True)
            future.result()
        else:
            future = self.theifForUrlResources(url, useHeader=True, forceUpdate=True)
            future.result()
        pass


def theifForPageInfo(self):
    #
    res = requests.get(
        "%s/merchant/product?"
        "pageSize=20&orderBy=&productCategoryId="
        "&productName=&tag=&pageIndex=0" % (self.baseweburl),
        headers=self.header);
    item = res.json()
    if item['resultCode'] != 200:
        raise Exception("cannot not read page info")
    data = item['data']
    return {
        'pages': data['pages'],  # 页大小
        'total': data['total'],  # 总条目
        'size': data['size'],  # 总页数
    }


if __name__ == "__main__":
    dt = DataTheif()
    # dt.theifForProductList()
    # dt.theifForProduct()
    # # info =dt.theifForPageInfo()
    # # size=info['size']
    # print(size)
    # dt.theifForProductList(1)
    # for i in range(,size):
    #     dt.theifForProductList(i)
    # dt.executor.shutdown(wait=True)

    # del dt;
    # dt.theifForUrlResources("https://merchant.qifengcm.com/merchant/product/10317")
    # dt.theifForUrlResources("https://aidi-public.oss-cn-zhangjiakou.aliyuncs.com/commodity/20200912/xSNrkHqL.jpg")
    #     /Volumes/Mac/Code/pycharm/MyAI/storebase/merchant/product
    #     https: // merchant.qifengcm.com / merchant / product / 10061

    # dt.theifForProductById(10319)

    # for i in range(10062, 10318):
    #     filename = "/Volumes/Mac/Code/pycharm/MyAI/storebase/merchant/product/%d" % (i)
    #     if not os.path.exists(filename):
    #         dt.theifForProductById(i)
    #         # logging.error("lack of%d"%(i))
    # dt.theifForCategory();
    items=['10350','10349','10348','10347','10346','10345','10344','10343','10342']
    for item in items:
        dt.theifForProductById(item)

